from bs4 import BeautifulSoup
import pymongo
from pymongo import MongoClient
import requests as req
import json
import datetime
from bson.objectid import ObjectId
from threading import Thread
from queue import Queue

start = 1

baseUrl = "https://patents.google.com/patent/"
#"/opt/oldPatentJson/patent.json"
inputJsonPath = "F:/专利数据/2018/20180201_20180229.json"
outputJsonPath = "F:/test.json"

client = pymongo.MongoClient('134.175.56.138', 27017)

class DownloadPatentFullText(Thread):
	def __init__(self, patentData, q):
		super(DownloadPatentFullText, self).__init__()
		self.patentData = patentData
		self.q = q
	
	def run(self):
		self.download_patent_fulltext()
	
	def download_patent_fulltext(self):
		try:
			a = 1 / 0
			res = req.get(baseUrl + self.patentData['publishNo'])
			res.encoding = 'UTF-8'
			soup = BeautifulSoup(res.text, "html.parser")
			patentFullText = soup.find('section', attrs={"itemprop": "description"}).find('div').text
			patentFullTextStr = "".join([line for line in patentFullText.split('\n') if line.strip() != ''])
			claim = soup.find('section', attrs={"itemprop": "claims"}).find('div').text
			claimStr = "".join([line for line in claim.split('\n') if line.strip() != ''])
			self.patentData['patentFullText'] = patentFullTextStr
			self.patentData['patentClaim'] = claimStr
			self.patentData['convertFlag'] = '0'
		except BaseException:
			self.patentData['convertFlag'] = '1'
		finally:
			self.q.put(self.patentData)
		
def saveProcessLog(log):
	db = client.patentFullText
	db.authenticate("root","lotut@mongodb")
	processLog = db.processLog
	processLog.insert_one(log)

def str2Json(str):
	data = json.loads(str)
	patentData = {}
	publishNo = data['publishNo']
	patentData["_id"] = data['_id']
	patentData["pid"] = data['pid']
	patentData["appNo"] = data['appNo']
	patentData["publishNo"] = publishNo
	patentData["publishDate"] = data['publishDate']
	patentData["patentType"] = data['patentType']
	return patentData;

def outputData(line):
	with open(outputJsonPath, 'a', encoding = "utf-8") as f:
		f.write(line)
		f.write('\n')

idx = 0
#源数据json
source_data_queue = Queue()
file = open(inputJsonPath, encoding = "utf-8")
line = file.readline()
while line:
	idx = idx + 1
	if idx >= start:
		source_data_queue.put(str2Json(line))
	line = file.readline()

Thread_list = []
target_data_source = Queue()
while not source_data_queue.empty():
	p = DownloadPatentFullText(source_data_queue.get(), target_data_source)
	p.start()
	Thread_list.append(p)
	
for i in Thread_list:
	i.join()

while not target_data_source.empty():
	outputData(json.dumps(target_data_source.get(), ensure_ascii=False))	
	
	

